package com.kd.crawler.common.utils;

import java.io.BufferedReader;
import java.io.IOException;  
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;  
import java.net.URISyntaxException;  
import java.nio.charset.Charset;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;  
import java.util.HashMap;
import java.util.List;
import java.util.Map;  
  

import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLException;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import org.apache.http.HttpEntity;  
import org.apache.http.HttpStatus;
import org.apache.http.HttpVersion;
import org.apache.http.NameValuePair;  
import org.apache.http.client.ClientProtocolException;  
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;  
import org.apache.http.client.methods.CloseableHttpResponse;  
import org.apache.http.client.methods.HttpGet;  
import org.apache.http.client.methods.HttpPost;  
import org.apache.http.client.methods.HttpRequestBase;  
import org.apache.http.client.utils.URIBuilder;  
import org.apache.http.config.ConnectionConfig;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.conn.ssl.X509HostnameVerifier;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;  
import org.apache.http.impl.client.HttpClients;  
  
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;  
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;  
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.TrustStrategy;
import org.apache.http.util.EntityUtils;  

import com.kd.crawler.http.HttpResponse;
import com.kd.utils.HtmlCleanerUtils;
  
/** 
 *  
 * @author Nan 2015-11 
 */  
public class HttpClientUtils {  
    private static PoolingHttpClientConnectionManager httpClientPool;  
//    private static String EMPTY_STR = "";  
    private static String UTF_8 = "UTF-8";  
  
    private static void init() {  
        if (null == httpClientPool) {
        	synchronized(HttpClientUtils.class) {
        		if(null == httpClientPool) {
        			Properties pro = PropertiesUtils.getProperties("crawl.properties");
        			int nHttpClients = 0;
        			nHttpClients = Integer.parseInt(pro.getProperty("thread_count", "32"));
		        	httpClientPool = new PoolingHttpClientConnectionManager(registry());
		        	//连接池持可持有多个HttpClient，HttpClient是处理请求的
		        	httpClientPool.setMaxTotal(nHttpClients);// 整个连接池最大连接数  
		        	httpClientPool.setDefaultMaxPerRoute(5);// 每路由最大连接数，默认值是2
        		}
        	}
        }  
    }  
  
    private static Registry<ConnectionSocketFactory> registry() {
    	 SSLContext ctx = null;
 		try {
 			X509TrustManager tm = new X509TrustManager() {
 				public void checkClientTrusted(X509Certificate[] arg0, String arg1)
 						throws CertificateException {
 				}

 				public void checkServerTrusted(X509Certificate[] arg0, String arg1)
 						throws CertificateException {
 					
 				}

 				public X509Certificate[] getAcceptedIssuers() {
 					return null;
 				}
 			};
 			ctx = SSLContext.getInstance("TLS");
 			ctx.init(null, new TrustManager[]{tm}, null);
 			Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
 	        		.register("http", PlainConnectionSocketFactory.INSTANCE)
 	        		.register("https", new SSLConnectionSocketFactory(ctx)).build();
 			return socketFactoryRegistry;
 		} catch (NoSuchAlgorithmException e1) {
 			e1.printStackTrace();
 		} catch (KeyManagementException e) {
 			e.printStackTrace();
 		}
 		return null;
    }
    
    /** 
     * 通过连接池获取HttpClient 
     *  
     * @return 
     */  
    private static CloseableHttpClient getHttpClient() {  
        init();
        //set time out
//        RequestConfig requestConfig = RequestConfig.custom()  
//    	        .setConnectTimeout(5000).setConnectionRequestTimeout(1000)  
//    	        .setSocketTimeout(5000).build();
        
//        Protocol.registerProtocol("https", 
//        		new Protocol("https", new MySSLSocketFactory(), 443));
        
       
        return HttpClients.custom().setConnectionManager(httpClientPool)
        		.build();  
    }
  
    private static final String phantomFileName = "phantomjs-2.1.1";
    public static String asyncHttpRequest(String url) {
    	Runtime rt = Runtime.getRuntime();
    	String dir = HttpClientUtils.class.getClassLoader().getResource(phantomFileName).getPath();
    	String phantomJsPath = dir + "/bin/phantomjs";
    	String codeJsPath = dir +"/bin/code.js";
    	//these statements is due to next statements where Runtime creates a proccess.
    	if(StringUtils.isNotBlank(codeJsPath) && codeJsPath.startsWith("/")) {
    		codeJsPath = codeJsPath.substring(1);
    	}
    	String html = "";
    	BufferedReader br = null;
    	try {
    		//here, it is very strange.
    		long startTime = System.currentTimeMillis();
			Process p = rt.exec(phantomJsPath + "  " + codeJsPath+" " + url);
			InputStream is = p.getInputStream();
			br = new BufferedReader(new InputStreamReader(is));
			StringBuffer sbf = new StringBuffer();
			String tmp = "";
			while((tmp = br.readLine())!=null) {
				sbf.append(tmp); 
			}
			html = sbf.toString();
			System.out.println(System.currentTimeMillis()-startTime);
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			if(null != br) {
				try {
					br.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
    	
    	return html;
    }
    /** 
     *  
     * @param url 
     * @return 
     */  
    public static HttpResponse httpGetRequest(String url) {  
        HttpGet httpGet = new HttpGet(url);  
        return getResult(httpGet);  
    }
  
    public static HttpResponse httpGetRequest(String url, Map<String, Object> params) throws URISyntaxException {  
        URIBuilder ub = new URIBuilder();  
        ub.setPath(url);  
  
        ArrayList<NameValuePair> pairs = covertParams2NVPS(params);  
        ub.setParameters(pairs);  
  
        HttpGet httpGet = new HttpGet(ub.build());  
        return getResult(httpGet);  
    }  
  
    public static HttpResponse httpGetRequest(String url, Map<String, Object> headers, Map<String, Object> params)  
            throws URISyntaxException {  
        URIBuilder ub = new URIBuilder();  
        ub.setPath(url);  
  
        if(null != params) {
	        ArrayList<NameValuePair> pairs = covertParams2NVPS(params);  
	        ub.setParameters(pairs);  
        }
        HttpGet httpGet = new HttpGet(ub.build());  
        for (Map.Entry<String, Object> param : headers.entrySet()) {
        	
//            httpGet.addHeader(param.getKey(), String.valueOf(param.getValue()));  
        	httpGet.addHeader(new BasicHeader(param.getKey(), String.valueOf(param.getValue())));  
        }  
        return getResult(httpGet);  
    }  
  
    public static HttpResponse httpPostRequest(String url) {  
        HttpPost httpPost = new HttpPost(url);  
        return getResult(httpPost);  
    }  
  
    public static HttpResponse httpPostRequest(String url, Map<String, Object> params) throws UnsupportedEncodingException {  
        HttpPost httpPost = new HttpPost(url);  
        ArrayList<NameValuePair> pairs = covertParams2NVPS(params);  
        httpPost.setEntity(new UrlEncodedFormEntity(pairs, UTF_8));  
        return getResult(httpPost);  
    }
  
    public static HttpResponse httpPostRequest(String url, Map<String, Object> headers, Map<String, Object> params)  
            throws UnsupportedEncodingException {  
        HttpPost httpPost = new HttpPost(url);  
  
        for (Map.Entry<String, Object> param : headers.entrySet()) {  
            httpPost.addHeader(param.getKey(), String.valueOf(param.getValue()));  
        }  
  
        if(null != params) {
	        ArrayList<NameValuePair> pairs = covertParams2NVPS(params);  
	        httpPost.setEntity(new UrlEncodedFormEntity(pairs, UTF_8));  
        }
        return getResult(httpPost);  
    }  
  
    private static ArrayList<NameValuePair> covertParams2NVPS(Map<String, Object> params) {
        ArrayList<NameValuePair> pairs = new ArrayList<NameValuePair>();  
        for (Map.Entry<String, Object> param : params.entrySet()) {  
            pairs.add(new BasicNameValuePair(param.getKey(), String.valueOf(param.getValue())));  
        }  
  
        return pairs;  
    }  
  
    /** 
     * 处理Http请求 
     *  
     * @param request 
     * @return 
     */  
    private static HttpResponse getResult(HttpRequestBase request) {  
        // CloseableHttpClient httpClient = HttpClients.createDefault();  
        CloseableHttpClient httpClient = getHttpClient();
        HttpResponse response = new HttpResponse();
        CloseableHttpResponse httpResponse = null;
        String result = "";
        try {  
            httpResponse = httpClient.execute(request);
            // response.getStatusLine().getStatusCode();  
            HttpEntity entity = httpResponse.getEntity();  
            if (null != entity) {  
                // long len = entity.getContentLength();// -1 表示长度未知  
            	byte[] bs = EntityUtils.toByteArray(entity);
                String charset = getContentCharset(new String(bs, "utf-8"));
                result = new String(bs, charset);
                if(isMessyCode(result)) {
                	result = new String(bs, "utf-8");
                }
            }
        } catch (ClientProtocolException e) {  
            e.printStackTrace();  
        } catch (IOException e) {  
            e.printStackTrace();  
        } finally {  
            try {
            	response.setHtml(result);
                response.setCrawlTime(System.currentTimeMillis()/1000);
            	if(null != httpResponse) {
            		response.setStatus(httpResponse.getStatusLine().getStatusCode());
            		httpResponse.close();
            	} else {
            		response.setStatus(404);
            	}
			} catch (IOException e) {
				e.printStackTrace();
			} catch (Exception e) {
				e.printStackTrace();
			}
            // httpClient.close();
        }  
        
        return response;
    }

  
    
    /**
     * according to a html, get its charset
     * @param headerHtml
     * @return
     */
    public static String getContentCharset(String headerHtml) {
        String charset = "ISO_8859-1";  
        String metaString= headerHtml;
        if (headerHtml != null) {  
        	//String metaString= headerHtml
        	Matcher m=Pattern.compile("<meta.*?>").matcher(headerHtml);
        	while(m.find()){
        		metaString = m.group(0);
        		 if (match(metaString, "(charset)\\s?=\\s?([\"]?utf-?8[\"]?)")) {  
                     charset = "utf-8";  
                     break;
                 } else if (match(metaString, "(charset)\\s?=\\s?([\"]?gbk[\"]?)")) {  
                     charset = "gbk";  
                     break;
                 } else if (match(metaString, "(charset)\\s?=\\s?([\"]?gb2312[\"]?)")) {  
                     charset = "gb2312";// http://ks.examda.com  
                     break;
                 }
        	}
        	
        }
        //reason 1: GBK supports gb2312;
        //reason 2: a charset gb2312 encodes a html where some complex char, such as'何旻熹'.
        // refer to 'http://shuhua.ce.cn/dtbf/201611/21/t20161121_17976276.shtml'
        if("gb2312".equals(charset)) {
        	charset = "gbk";
        }
        return charset;  
    }
 
	public static boolean match(String s, String pattern) {  
        Pattern p = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE + Pattern.UNICODE_CASE);  
        Matcher matcher = p.matcher(s);  
        if (matcher.find()) {  
            return true;  
        } else {  
            return false;  
        }  
    } 
  
	 /**
     * 判断字符是否是中文
     *
     * @param c 字符
     * @return 是否是中文
     */
    public static boolean isChinese(char c) {
        Character.UnicodeBlock ub = Character.UnicodeBlock.of(c);
        if (ub == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS
                || ub == Character.UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS
                || ub == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A
                || ub == Character.UnicodeBlock.GENERAL_PUNCTUATION
                || ub == Character.UnicodeBlock.CJK_SYMBOLS_AND_PUNCTUATION
                || ub == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS) {
            return true;
        }
        return false;
    }
	/**
     * 判断字符串是否是乱码
     *
     * @param strName 字符串
     * @return 是否是乱码
     */
    public static boolean isMessyCode(final String html) {
        Pattern p = Pattern.compile("\\s*|t*|r*|n*");
        Matcher m = p.matcher(HtmlCleanerUtils.getTextFromHtml(html));
        String after = m.replaceAll("");
        String temp = after.replaceAll("\\p{P}", "");
        char[] ch = temp.trim().toCharArray();
        float chLength = ch.length;
        float count = 0;
        for (int i = 0; i < ch.length; i++) {
            char c = ch[i];
            if (!Character.isLetterOrDigit(c)) {
                if (!isChinese(c)) {
                    count = count + 1;
                }
            }
        }
        float result = count / chLength;
        if (result > 0.1) {
            return true;
        } else {
            return false;
        }
 
    }
    public static void main(String[] args) {
//    	String url = "http://www.tj0454.com/life/list.htm?classid=55";
    	String url = "http://shop.tianya.cn/my/orders";
    	Map<String, Object> map = new HashMap<String, Object>();
    	String value= "time=ct=1481074932.909; __guid=1439947928; __ptime=1481074939711; ADVC=349cc48283af3c; ASL=17142,00kqj,0e7f1e460e7f1e0f0e7f1e0f; "
    			+ "__cid=57; __cfduid=d5b6493ff4aa5d0a0bc2729c1d432c4c61480906660; "
    			+ "Hm_lvt_80579b57bf1b16bdf88364b13221a8bd=1481013630,1481013664,1481013789,1481074839; __auc=ba378e33158cce960e9b551182c; "
    			+ "__guid2=1439947928; Hm_lvt_bc5755e0609123f78d0e816bf7dee255=1480906723,1481013615; bbs_msg=1481074901511_122320045_0_0_0_0; "
    			+ "ty_msg=1481074901335_122320045_2_0_0_0_0_0_2_0_0_0_0; bc_ids_m=lq; tianyaATC=tianya,3111199,20958,21140,1481017367972,bbs.tianya.cn%2F; "
    			+ "tianya1=120121,1481017367,2,83158,18768,1481018012,2,85756,2980,1481018003,2,85814; "
    			+ "Hm_lvt_6654fe18597114a838bf895bc52ed9f2=1480906950,1481017300; bc_ids_w=mu_n0; ADVS=349dbdac9c624f; "
    			+ "Hm_lpvt_80579b57bf1b16bdf88364b13221a8bd=1481074839; u_tip=122320045=0; Hm_lpvt_bc5755e0609123f78d0e816bf7dee255=1481074934; "
    			+ "vip=955660640%3D0; __visit=wpopa%3D1262; vk=95a5cbf012a03e5f; bc_ids_d=od_oe_oa; Hm_lvt_4b210818e48a062efe31b92c7d036965=1481015818; "
    			+ "Hm_lpvt_4b210818e48a062efe31b92c7d036965=1481073938; uvhb=1481017295817.1; Hm_lpvt_6654fe18597114a838bf895bc52ed9f2=1481017939; "
    			+ "__u_a=v2.2.2; __asc=d61405e9158d6f09086a046e7cb; bc_exp=2; "
    			+ "temp=k=625740102&s=&t=1481075034&b=1d0c368c13df5657c3bd32c2d5906d0d&ct=1481075034&et=1483667034; "
    			+ "sso=r=1295205079&sid=&wsid=F3EA2331D77BC0CB571060BA4720B0C4; user=w=micdp&id=122320045&f=1; right=web4=n&portal=n; "
    			+ "temp4=rm=2799aa6e65795b5aafb7d33d79d07c22";
//    	map.put("Cookie", value);
    	map.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0");
//		try {
//			System.out.println(httpGetRequest(url, map, null).getHtml());
//		} catch (URISyntaxException e) {
//			e.printStackTrace();
//		}
//		try {
//			System.out.println(httpPostRequest(url, map, null).getHtml());
//		} catch (UnsupportedEncodingException e) {
//			e.printStackTrace();
//		}
    	
//		System.out.println(httpPostRequest("http://baoliao.cq.qq.com/pc/detail.html?id=294064").getHtml());
    	
//    	System.out.println(isMessyCode("Ã©Å¸Â©Ã©Â¡ÂºÃ¥Â¹Â³"));
//    	System.out.println(Character.i('a'));
//        System.out.println(isMessyCode("你好"));
    	
    	//test for async
//    	System.out.println("async http request: " + asyncHttpRequest("http://baoliao.cq.qq.com/pc/detail.html?id=294064"));
    	
    	System.out.println(httpGetRequest("https://www.hndnews.com/p/3406.html").getHtml());
    	
    }
}  